{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cefda1e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "SOURCE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__name__))))\n",
    "SOURCE_DIR\n",
    "sys.path.insert(0, SOURCE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.23 s, sys: 3.35 s, total: 6.58 s\n",
      "Wall time: 2.39 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<module 'malaya' from '/home/husein/dev/malaya/malaya/__init__.py'>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import malaya\n",
    "malaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm = malaya.language_model.kenlm(model = 'bahasa-wiki-news')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-09 17:01:13.967314: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-12-09 17:01:13.971794: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
      "2022-12-09 17:01:13.971817: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: husein-MS-7D31\n",
      "2022-12-09 17:01:13.971822: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: husein-MS-7D31\n",
      "2022-12-09 17:01:13.971902: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: Not found: was unable to find libcuda.so DSO loaded into this program\n",
      "2022-12-09 17:01:13.971922: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 470.161.3\n"
     ]
    }
   ],
   "source": [
    "stemmer = malaya.stem.deep_model(model = 'noisy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrector = malaya.spelling_correction.probability.load(language_model = lm, replace_augmentation = True,\n",
    "                                                       stemmer = stemmer, maxlen = 12, minlen = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrector._replace_augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from malaya.spelling_correction.base import _augment_vowel_alternate, norvig_method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 25 µs, sys: 23 µs, total: 48 µs\n",
      "Wall time: 50.1 µs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['mndptkn']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "corrector.edit_candidates('mndptkn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 113 ms, sys: 7.74 ms, total: 121 ms\n",
      "Wall time: 121 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['kemik',\n",
       " 'kecik',\n",
       " 'decik',\n",
       " 'gecik',\n",
       " 'kecek',\n",
       " 'kecak',\n",
       " 'kerik',\n",
       " 'kepik',\n",
       " 'ketik',\n",
       " 'kecok',\n",
       " 'kecil',\n",
       " 'kedik',\n",
       " 'recik',\n",
       " 'kenik',\n",
       " 'keciak',\n",
       " 'kechik',\n",
       " 'kesik',\n",
       " 'kelik']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "corrector.edit_candidates('kecik')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 480 µs, sys: 437 µs, total: 917 µs\n",
      "Wall time: 922 µs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['tenpat', 'tempat', 'tengat']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "corrector.edit_candidates('tenpat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9 µs, sys: 9 µs, total: 18 µs\n",
      "Wall time: 20.3 µs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "len(corrector.edit_step('mndptkn'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer = malaya.normalize.normalizer(corrector, stemmer, date = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "segmenter = malaya.segmentation.huggingface()\n",
    "segmenter_func = lambda x: segmenter.generate([x], max_length = 256)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a T5TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 470 ms, sys: 0 ns, total: 470 ms\n",
      "Wall time: 41.9 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'tak kisah'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "segmenter_func('takkisah')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "fasttext = malaya.language_detection.fasttext()\n",
    "lang_word_model = malaya.language_detection.substring_rules(model = fasttext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-10.182438850402832"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score('Takkisah')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-10.182438850402832"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score('takkisah')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-12.641404151916504"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score('ayam sedap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-11.60669994354248"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score('Ayam sedap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-9.186513900756836"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score('najib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5.1223626136779785"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score('Najib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!rm *.normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "rm: cannot remove '2022-06-08-bahasa-17*': No such file or directory\n"
     ]
    }
   ],
   "source": [
    "!rm 2022-06-08-bahasa-17*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "\n",
    "twitters = glob('/home/husein/malaya/2022-06-08/*.json')\n",
    "twitters.extend(glob('/home/husein/malaya/compiled/*.json'))\n",
    "twitters = sorted(twitters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/husein/malaya/2022-06-08/2022-06-08-bahasa-0.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-1.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-10.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-11.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-12.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-13.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-14.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-15.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-16.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-17.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-18.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-19.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-2.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-20.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-21.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-22.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-23.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-24.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-25.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-3.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-4.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-5.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-6.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-7.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-8.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-9.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-0.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-1.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-10.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-11.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-12.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-13.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-14.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-15.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-16.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-17.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-18.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-19.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-2.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-20.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-21.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-22.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-23.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-24.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-25.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-26.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-27.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-28.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-29.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-3.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-30.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-31.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-32.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-33.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-34.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-35.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-36.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-37.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-38.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-39.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-4.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-40.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-41.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-5.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-6.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-7.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-8.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-9.json']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected_words = ['sik', 'aktivitas', 'lanat', 'keren', 'loh', 'akh', 'pake', 'goblok', 'ivo',\n",
    "                 'listrik', 'ludes', 'pemudik', 'stasiun', 'ketidaknyamananya', 'gawe', 'jempol',\n",
    "                 'kesahatan', 'sivitas', 'ngasih', 'pasokan', 'esdm', 'miliar', 'domisili',\n",
    "                 'nyengine', 'indonesia', 'ndak', 'mbak', 'gimana', 'verif']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "def query(text, from_lang = 'en', to_lang = 'ms'):\n",
    "    return requests.get(f'http://100.105.246.81:8999/api?from={from_lang}&to={to_lang}&text={text}').json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_scorer = lambda x: lm.score(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def fix_spacing_punct(string):\n",
    "    string = string.replace(' , ', ', ').replace(' !', '!').replace(' . ', '. ').replace('( ', '(') \\\n",
    "    .replace(' )', ')')\n",
    "    return re.sub(r'[ ]+', ' ', string).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-0.json 2022-06-08-bahasa-0.json.normalized 2022-06-08-bahasa-0.normalized.jsonl.requested\n",
      "skip 2022-06-08-bahasa-0.json\n",
      "2022-06-08-bahasa-1.json 2022-06-08-bahasa-1.json.normalized 2022-06-08-bahasa-1.normalized.jsonl.requested\n",
      "skip 2022-06-08-bahasa-1.json\n",
      "2022-06-08-bahasa-10.json 2022-06-08-bahasa-10.json.normalized 2022-06-08-bahasa-10.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [36:44<00:00, 45.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-11.json 2022-06-08-bahasa-11.json.normalized 2022-06-08-bahasa-11.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [35:01<00:00, 47.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-12.json 2022-06-08-bahasa-12.json.normalized 2022-06-08-bahasa-12.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [34:44<00:00, 47.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-13.json 2022-06-08-bahasa-13.json.normalized 2022-06-08-bahasa-13.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [35:23<00:00, 47.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-14.json 2022-06-08-bahasa-14.json.normalized 2022-06-08-bahasa-14.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [34:24<00:00, 48.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-15.json 2022-06-08-bahasa-15.json.normalized 2022-06-08-bahasa-15.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [35:16<00:00, 47.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-16.json 2022-06-08-bahasa-16.json.normalized 2022-06-08-bahasa-16.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [36:21<00:00, 45.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-17.json 2022-06-08-bahasa-17.json.normalized 2022-06-08-bahasa-17.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [36:42<00:00, 45.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-18.json 2022-06-08-bahasa-18.json.normalized 2022-06-08-bahasa-18.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [43:02<00:00, 38.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-19.json 2022-06-08-bahasa-19.json.normalized 2022-06-08-bahasa-19.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [41:51<00:00, 39.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-2.json 2022-06-08-bahasa-2.json.normalized 2022-06-08-bahasa-2.normalized.jsonl.requested\n",
      "skip 2022-06-08-bahasa-2.json\n",
      "2022-06-08-bahasa-20.json 2022-06-08-bahasa-20.json.normalized 2022-06-08-bahasa-20.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [45:00<00:00, 37.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-21.json 2022-06-08-bahasa-21.json.normalized 2022-06-08-bahasa-21.normalized.jsonl.requested\n",
      "skip 2022-06-08-bahasa-21.json\n",
      "2022-06-08-bahasa-22.json 2022-06-08-bahasa-22.json.normalized 2022-06-08-bahasa-22.normalized.jsonl.requested\n",
      "skip 2022-06-08-bahasa-22.json\n",
      "2022-06-08-bahasa-23.json 2022-06-08-bahasa-23.json.normalized 2022-06-08-bahasa-23.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [38:48<00:00, 42.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-24.json 2022-06-08-bahasa-24.json.normalized 2022-06-08-bahasa-24.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [41:23<00:00, 40.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-25.json 2022-06-08-bahasa-25.json.normalized 2022-06-08-bahasa-25.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 79802/79802 [30:42<00:00, 43.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-3.json 2022-06-08-bahasa-3.json.normalized 2022-06-08-bahasa-3.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [33:05<00:00, 50.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-4.json 2022-06-08-bahasa-4.json.normalized 2022-06-08-bahasa-4.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [32:51<00:00, 50.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-5.json 2022-06-08-bahasa-5.json.normalized 2022-06-08-bahasa-5.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [32:18<00:00, 51.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-6.json 2022-06-08-bahasa-6.json.normalized 2022-06-08-bahasa-6.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [28:38<00:00, 58.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-7.json 2022-06-08-bahasa-7.json.normalized 2022-06-08-bahasa-7.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [30:32<00:00, 54.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-8.json 2022-06-08-bahasa-8.json.normalized 2022-06-08-bahasa-8.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [30:30<00:00, 54.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-9.json 2022-06-08-bahasa-9.json.normalized 2022-06-08-bahasa-9.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [31:42<00:00, 52.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-0.json 2021-dump-bahasa-twitter-0.json.normalized 2021-dump-bahasa-twitter-0.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [31:30<00:00, 52.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-1.json 2021-dump-bahasa-twitter-1.json.normalized 2021-dump-bahasa-twitter-1.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [31:44<00:00, 52.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-10.json 2021-dump-bahasa-twitter-10.json.normalized 2021-dump-bahasa-twitter-10.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [30:38<00:00, 54.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-11.json 2021-dump-bahasa-twitter-11.json.normalized 2021-dump-bahasa-twitter-11.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [34:19<00:00, 48.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-12.json 2021-dump-bahasa-twitter-12.json.normalized 2021-dump-bahasa-twitter-12.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [34:08<00:00, 48.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-13.json 2021-dump-bahasa-twitter-13.json.normalized 2021-dump-bahasa-twitter-13.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [33:57<00:00, 49.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-14.json 2021-dump-bahasa-twitter-14.json.normalized 2021-dump-bahasa-twitter-14.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [29:16<00:00, 56.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-15.json 2021-dump-bahasa-twitter-15.json.normalized 2021-dump-bahasa-twitter-15.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [26:18<00:00, 63.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-16.json 2021-dump-bahasa-twitter-16.json.normalized 2021-dump-bahasa-twitter-16.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [31:05<00:00, 53.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-17.json 2021-dump-bahasa-twitter-17.json.normalized 2021-dump-bahasa-twitter-17.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [32:54<00:00, 50.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-18.json 2021-dump-bahasa-twitter-18.json.normalized 2021-dump-bahasa-twitter-18.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [30:15<00:00, 55.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-19.json 2021-dump-bahasa-twitter-19.json.normalized 2021-dump-bahasa-twitter-19.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|█████████████████                                                                 | 20763/100000 [06:47<18:38, 70.81it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [31:12<00:00, 53.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-21.json 2021-dump-bahasa-twitter-21.json.normalized 2021-dump-bahasa-twitter-21.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|█████████████████▏                                                                | 21030/100000 [07:43<41:18, 31.86it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [32:07<00:00, 51.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-22.json 2021-dump-bahasa-twitter-22.json.normalized 2021-dump-bahasa-twitter-22.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██████████████████████                                                            | 26924/100000 [09:14<18:07, 67.18it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████| 100000/100000 [26:28<00:00, 62.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-28.json 2021-dump-bahasa-twitter-28.json.normalized 2021-dump-bahasa-twitter-28.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [27:43<00:00, 60.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-29.json 2021-dump-bahasa-twitter-29.json.normalized 2021-dump-bahasa-twitter-29.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [30:00<00:00, 55.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-3.json 2021-dump-bahasa-twitter-3.json.normalized 2021-dump-bahasa-twitter-3.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [31:57<00:00, 52.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-30.json 2021-dump-bahasa-twitter-30.json.normalized 2021-dump-bahasa-twitter-30.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [31:07<00:00, 53.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-31.json 2021-dump-bahasa-twitter-31.json.normalized 2021-dump-bahasa-twitter-31.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [24:34<00:00, 67.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-32.json 2021-dump-bahasa-twitter-32.json.normalized 2021-dump-bahasa-twitter-32.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [26:17<00:00, 63.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-33.json 2021-dump-bahasa-twitter-33.json.normalized 2021-dump-bahasa-twitter-33.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [26:41<00:00, 62.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-34.json 2021-dump-bahasa-twitter-34.json.normalized 2021-dump-bahasa-twitter-34.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [29:59<00:00, 55.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-35.json 2021-dump-bahasa-twitter-35.json.normalized 2021-dump-bahasa-twitter-35.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [28:05<00:00, 59.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-36.json 2021-dump-bahasa-twitter-36.json.normalized 2021-dump-bahasa-twitter-36.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [26:18<00:00, 63.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-37.json 2021-dump-bahasa-twitter-37.json.normalized 2021-dump-bahasa-twitter-37.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [24:57<00:00, 66.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-38.json 2021-dump-bahasa-twitter-38.json.normalized 2021-dump-bahasa-twitter-38.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [24:52<00:00, 66.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-39.json 2021-dump-bahasa-twitter-39.json.normalized 2021-dump-bahasa-twitter-39.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [28:15<00:00, 58.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-4.json 2021-dump-bahasa-twitter-4.json.normalized 2021-dump-bahasa-twitter-4.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [28:55<00:00, 57.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-40.json 2021-dump-bahasa-twitter-40.json.normalized 2021-dump-bahasa-twitter-40.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [31:32<00:00, 52.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-41.json 2021-dump-bahasa-twitter-41.json.normalized 2021-dump-bahasa-twitter-41.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [29:27<00:00, 56.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-5.json 2021-dump-bahasa-twitter-5.json.normalized 2021-dump-bahasa-twitter-5.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 100000/100000 [31:09<00:00, 53.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-6.json 2021-dump-bahasa-twitter-6.json.normalized 2021-dump-bahasa-twitter-6.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|████████████▏                       | 33817/100000 [09:29<22:40, 48.64it/s]"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import traceback\n",
    "import json\n",
    "\n",
    "for f in twitters:\n",
    "    filename = os.path.split(f)[1]\n",
    "    output = f'{filename}.normalized'\n",
    "    output_requested = f\"{filename.replace('.json', '')}.normalized.jsonl.requested\"\n",
    "    print(filename, output, output_requested)\n",
    "    \n",
    "    if os.path.exists(output_requested):\n",
    "        print(f'skip {filename}')\n",
    "        continue\n",
    "    \n",
    "    with open(f) as fopen:\n",
    "        data = json.load(fopen)\n",
    "\n",
    "    results = []\n",
    "    already_in = set()\n",
    "    for row in tqdm(data):\n",
    "        if row['retweet_text_full'] == 'NULL':\n",
    "            t = row['data_text']\n",
    "        else:\n",
    "            t = row['retweet_text_full']\n",
    "\n",
    "        if fasttext.predict([t])[0] not in ['malay', 'rojak']:\n",
    "            continue\n",
    "\n",
    "        if t.lower() in already_in:\n",
    "            continue\n",
    "\n",
    "        tokenized = normalizer._tokenizer(t.lower())\n",
    "        if any([w in tokenized for w in rejected_words]):\n",
    "            continue\n",
    "\n",
    "        if not len(t):\n",
    "            continue\n",
    "\n",
    "        try:\n",
    "            normalized = normalizer.normalize(\n",
    "                t, normalize_hingga = False, normalize_cardinal = False,\n",
    "                normalize_ordinal = False, normalize_pada_hari_bulan = False,\n",
    "                normalize_fraction = False, normalize_money = False, normalize_date = False,\n",
    "                normalize_time = False, normalize_ic = False,\n",
    "                normalize_units = False,\n",
    "                normalize_url = False, normalize_percent=False, normalize_telephone = False,\n",
    "                text_scorer = text_scorer, segmenter = segmenter_func,\n",
    "                not_a_word_threshold = 1e-9,\n",
    "            )['normalize']\n",
    "            \n",
    "            normalized = fix_spacing_punct(normalized)\n",
    "            \n",
    "        except Exception as e:\n",
    "            print(traceback.format_exc())\n",
    "            print(e)\n",
    "            normalized = t\n",
    "        \n",
    "        results.append({'original': t, 'normalized': normalized})\n",
    "        already_in.add(t.lower())\n",
    "    \n",
    "    with open(output, 'w') as fopen:\n",
    "        json.dump(results, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fix_spacing_punct(results[5]['normalized'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer._tokenizer('RM35.92 . Get')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = malaya.tokenizer.Tokenizer(parliament = True).tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer('RM35.92 . Get')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer('RM35.92. Get')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
